package etxt2db.mallet;
/* Copyright (C) 2002 Univ. of Massachusetts Amherst, Computer Science Dept.
   This file is part of "MALLET" (MAchine Learning for LanguagE Toolkit).
   http://www.cs.umass.edu/~mccallum/mallet
   This software is provided under the terms of the Common Public License,
   version 1.0, as published by http://www.opensource.org.  For further
   information, see the file `LICENSE' included with this distribution. */


/**
   @author Ryan McDonald <a href="mailto:ryantm@cis.upenn.edu">ryantm@cis.upenn.edu</a>
 */

/*
	An error?  CoNLLTrue MalletTrue MalletPred
	O O O
	I-MISC B-MISC B-MISC
	B-MISC B-MISC I-MISC
	I-MISC B-MISC I-MISC
	O O O
	O O O
	O O O
*/

//package edu.umass.cs.mallet.users.ryantm.medline;

import java.util.StringTokenizer;

import edu.umass.cs.mallet.base.pipe.Pipe;
import edu.umass.cs.mallet.base.types.Instance;
import edu.umass.cs.mallet.base.types.LabelAlphabet;
import edu.umass.cs.mallet.base.types.LabelSequence;
import edu.umass.cs.mallet.base.types.Token;
import edu.umass.cs.mallet.base.types.TokenSequence;

public class MedlineSentence2TokenSequence extends Pipe
{

	/*
    static final String[] endings = new String[]
    {"ing", "ed", "ogy", "s", "ly", "ion", "tion", "ity", "ies"};
    static Pattern[] endingPatterns = new Pattern[endings.length];
    // Indexed by {forward,backward} {0,1,2 offset} {ending char ngram index}
    static final String[][][] endingNames = new String[2][3][endings.length];

    {
	for (int i = 0; i < endings.length; i++) {
	    endingPatterns[i] = Pattern.compile (".*"+endings[i]+"$");
	    for (int j = 0; j < 3; j++) {
		for (int k = 0; k < 2; k++)
		    endingNames[k][j][i] = "W"+(k==1?"-":"")+j+"=<END"+endings[i]+">";
	    }
	}
    }
    */

    /**
	 * 
	 */
	private static final long serialVersionUID = -4159669184789849503L;
	boolean saveSource = false;
    boolean doSpelling = false;
    boolean doDigitCollapses = true;
    boolean doDowncasing = true;
    
    public MedlineSentence2TokenSequence ()
    {
	super (null, LabelAlphabet.class);
    }
    
    public MedlineSentence2TokenSequence (boolean extraFeatures)
    {
	super (null, LabelAlphabet.class);
	if (!extraFeatures) {
	    doDigitCollapses = doSpelling = false;
	    doDowncasing = true;
	}
    }
    
    /* Lines look like this:
       
       allele O
       - O
       specific O
       for O
       activating O
       mutations B-type
       at O
       the O
       12th B-location
       , O
       13th B-location
       , O
       and O
       codons B-location
       of O
       three O
	
    */
    
    public Instance pipe (Instance carrier)
    {
	String sentenceLines = (String) carrier.getData();
	String[] tokens = sentenceLines.split ("\n");
	TokenSequence data = new TokenSequence (tokens.length);
	LabelSequence target = new LabelSequence ((LabelAlphabet)getTargetAlphabet(), tokens.length);

	/*
	boolean [][] ending = new boolean[3][endings.length];
	boolean [][] endingp1 = new boolean[3][endings.length];
	boolean [][] endingp2 = new boolean[3][endings.length];
	*/
	
	StringBuffer source = saveSource ? new StringBuffer() : null;
	
	//Pattern ipattern = Pattern.compile ("I-.*");
	String word, label;
	for (int i = 0; i < tokens.length; i++) {
	    if (tokens[i].length() != 0) {
		//String[] features = tokens[i].split(" ");
		StringTokenizer st = new StringTokenizer(tokens[i]);
		label = st.nextToken();
		word = st.nextToken();
		//System.out.println("label = "+label+" word = "+word);
		/*if (features.length != 2)
		    throw new IllegalStateException ("Line \""+tokens[i]+"\" doesn't have 2 elements");
		*/
		//word = features[0]; // .toLowerCase();
		//label = features[1];
	    } else {
		word = "-<S>-";
		label = "O";
	    }
	    
	    // Transformations
	    if (doDigitCollapses) {
		if (word.matches ("19\\d\\d"))
		    word = "<YEAR>";
		else if (word.matches ("19\\d\\ds"))
		    word = "<YEARDECADE>";
		else if (word.matches ("19\\d\\d-\\d+"))
		    word = "<YEARSPAN>";
		else if (word.matches ("\\d+\\\\/\\d"))
		    word = "<FRACTION>";
		else if (word.matches ("\\d[\\d,\\.]*"))
		    word = "<DIGITS>";
		else if (word.matches ("19\\d\\d-\\d\\d-\\d--d"))
		    word = "<DATELINEDATE>";
		else if (word.matches ("19\\d\\d-\\d\\d-\\d\\d"))
		    word = "<DATELINEDATE>";
		else if (word.matches (".*-led"))
		    word = "<LED>";
		else if (word.matches (".*-sponsored"))
		    word = "<LED>";
	    }
	    
	    if (doDowncasing)
		word = word.toLowerCase();

	    Token token = new Token (word);
	    
	    /*
	    // Word and tag unigram at current time
	    if (doSpelling) {
		for (int j = 0; j < endings.length; j++) {
		    ending[2][j] = ending[1][j];
		    ending[1][j] = ending[0][j];
		    ending[0][j] = endingPatterns[j].matcher(word).matches();
		    if (ending[0][j]) token.setFeatureValue (endingNames[0][0][j], 1);
		}
	    }
	    */

	    // Append
	    data.add (token);
	    //target.add (bigramLabel);
	    target.add (label);
	    //System.out.print (label + ' ');
	    if (saveSource) {
		source.append (word); 
		source.append (" ");
				//source.append (bigramLabel); source.append ("\n");
		source.append (label); source.append ("\n");
	    }
	    
	}

	carrier.setData(data);
	carrier.setTarget(target);
	if (saveSource)
	    carrier.setSource(source);
	return carrier;
    }
}
